{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "975ed346",
   "metadata": {},
   "source": [
    "# 自定义评价指标\n",
    "当模型训练完成后，需要一个评价指标来评价模型的好坏，常用的评价指标有准确率、精确率、召回率等。"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "668e2e65",
   "metadata": {},
   "source": [
    "我们先复制一下之前的代码："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "58810f42",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WARNING] ME(27752:3544,MainProcess):2022-10-17-09:22:59.199.665 [mindspore\\dataset\\engine\\datasets_user_defined.py:656] Python multiprocessing is not supported on Windows platform.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 11.307688\n",
      "Loss: 6.7457876\n",
      "Loss: 9.408869\n",
      "Loss: 8.844019\n",
      "Loss: 7.620488\n",
      "Loss: 7.3819017\n",
      "Loss: 3.9230263\n",
      "Loss: 4.9625993\n",
      "Loss: 2.8809724\n",
      "Loss: 2.4225113\n",
      "Loss: 3.8251824\n",
      "Loss: 2.4700258\n",
      "Loss: 6.5675097\n",
      "Loss: 2.4831178\n",
      "Loss: 5.4834557\n",
      "Loss: 5.815234\n",
      "Loss: 5.17356\n",
      "Loss: 3.573363\n",
      "Loss: 3.2425675\n",
      "Loss: 3.581607\n",
      "Loss: 2.628729\n",
      "Loss: 2.3038902\n",
      "Loss: 2.2710965\n",
      "Loss: 2.146978\n",
      "Loss: 2.3615096\n",
      "Loss: 3.2800179\n",
      "Loss: 1.937361\n",
      "Loss: 3.000183\n",
      "Loss: 4.1199512\n",
      "Loss: 2.0758872\n",
      "Loss: 1.952487\n",
      "Loss: 2.89281\n",
      "Loss: 1.9690603\n",
      "Loss: 2.0625663\n",
      "Loss: 2.2853744\n",
      "Loss: 2.662634\n",
      "Loss: 2.2194324\n",
      "Loss: 2.0476727\n",
      "Loss: 1.6774727\n",
      "Loss: 1.7701403\n",
      "Loss: 1.7973168\n",
      "Loss: 1.774527\n",
      "Loss: 1.5038054\n",
      "Loss: 1.3892232\n",
      "Loss: 1.9401461\n",
      "Loss: 1.8018448\n",
      "Loss: 1.3755263\n",
      "Loss: 1.152633\n",
      "Loss: 1.7630688\n",
      "Loss: 0.8617687\n",
      "Loss: 1.417712\n",
      "Loss: 1.1014007\n",
      "Loss: 0.59174544\n",
      "Loss: 0.9952431\n",
      "Loss: 1.0153236\n",
      "Loss: 1.4480485\n",
      "Loss: 1.1746943\n",
      "Loss: 0.8712713\n",
      "Loss: 0.79514456\n",
      "Loss: 0.41021934\n",
      "Loss: 0.9138632\n",
      "Loss: 0.8997561\n",
      "Loss: 0.9829502\n",
      "Loss: 0.64047384\n",
      "Loss: 0.64007777\n",
      "Loss: 0.60184324\n",
      "Loss: 1.2909706\n",
      "Loss: 0.6420765\n",
      "Loss: 0.47053584\n",
      "Loss: 0.6665419\n",
      "Loss: 0.8895152\n",
      "Loss: 0.753878\n",
      "Loss: 0.8600809\n",
      "Loss: 0.70659196\n",
      "Loss: 0.6985352\n",
      "Loss: 0.76514226\n",
      "Loss: 0.66382366\n",
      "Loss: 0.5475451\n",
      "Loss: 0.7986258\n",
      "Loss: 1.1214272\n",
      "Loss: 0.7455339\n",
      "Loss: 0.9521564\n",
      "Loss: 0.88271224\n",
      "Loss: 1.3937421\n",
      "Loss: 0.7669326\n",
      "Loss: 0.6697707\n",
      "Loss: 0.8908321\n",
      "Loss: 0.5953002\n",
      "Loss: 0.7858795\n",
      "Loss: 1.5519414\n",
      "Loss: 1.2663016\n",
      "Loss: 0.90012074\n",
      "Loss: 0.8677365\n",
      "Loss: 0.8783139\n",
      "Loss: 1.3167031\n",
      "Loss: 1.0902796\n",
      "Loss: 0.84668666\n",
      "Loss: 0.9685418\n",
      "Loss: 1.1742703\n",
      "Loss: 0.7151176\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import mindspore.dataset as ds\n",
    "import mindspore.nn as nn\n",
    "import mindspore.ops as ops\n",
    "import mindspore as ms\n",
    "\n",
    "\n",
    "def create_data(data, w=2.0, b=3.0):\n",
    "    for item in range(data):\n",
    "        # 使用uniform均匀分布数据\n",
    "        x = np.random.uniform(-10.0, 10.0)\n",
    "        # 指定一个噪声来模拟数据\n",
    "        # 使用normal使其满足高斯分布,参数为数据的中间值和浮动值\n",
    "        noise = np.random.normal(0, 1)\n",
    "        y = w * x + b + noise\n",
    "        # 使用yield返回数据生成器，方便后续对数据进行处理\n",
    "        yield np.array([x]).astype(np.float32), np.array([y]).astype(np.float32)\n",
    "\n",
    "\n",
    "def create_dataset(data_num, batch_size, repeat_num):\n",
    "    dataset = ds.GeneratorDataset(list(create_data(data_num)), column_names=['data', 'label'])\n",
    "    dataset = dataset.batch(batch_size, drop_remainder=True)\n",
    "    dataset = dataset.repeat(repeat_num)\n",
    "    return dataset\n",
    "\n",
    "\n",
    "# 创建训练数据集\n",
    "train_dataset = create_dataset(data_num=1000, batch_size=10, repeat_num=1)\n",
    "\n",
    "\n",
    "class MyNet(nn.Cell):\n",
    "    \"\"\"定义网络模型\"\"\"\n",
    "    def __init__(self):\n",
    "        super(MyNet, self).__init__()\n",
    "        self.layer = nn.Dense(1, 1)\n",
    "\n",
    "    def construct(self, x):\n",
    "        fx = self.layer(x)\n",
    "        return fx\n",
    "\n",
    "\n",
    "# 初始化网络模型\n",
    "net = MyNet()\n",
    "\n",
    "# 获取可训练参数w和b\n",
    "model_params = net.trainable_params()\n",
    "\n",
    "\n",
    "class MyLossFunction(nn.LossBase):\n",
    "    \"\"\"定义损失函数\"\"\"\n",
    "    def __init__(self):\n",
    "        super(MyLossFunction, self).__init__()\n",
    "        self.abs = ops.Abs()\n",
    "\n",
    "    def construct(self, target, predict):\n",
    "        x = self.abs(target - predict)\n",
    "        return self.get_loss(x)\n",
    "\n",
    "\n",
    "class MyOptimizer(nn.Optimizer):\n",
    "    \"\"\"自定义优化器\"\"\"\n",
    "    def __init__(self, learning_rate, params, momentum=0.9):\n",
    "        super(MyOptimizer, self).__init__(learning_rate, params)\n",
    "        self.assign = ops.Assign()\n",
    "        # 定义初速度\n",
    "        self.moment = ms.Parameter(ms.Tensor(momentum, ms.float32), name=\"moment\")\n",
    "        # 定义动量参数\n",
    "        self.momentum = self.parameters.clone(prefix=\"momentum\", init=\"zeros\")\n",
    "\n",
    "    def construct(self, gradients):\n",
    "        lr = self.get_lr()\n",
    "        params = self.parameters\n",
    "        for i in range(len(params)):\n",
    "            # 给网络赋值动量参数\n",
    "            self.assign(self.momentum[i], self.momentum[i] * self.moment + gradients[i])\n",
    "            # 计算新的权重参数\n",
    "            update = params[i] - self.momentum[i] * lr\n",
    "            # 更新权重参数\n",
    "            self.assign(params[i], update)\n",
    "        return params\n",
    "\n",
    "\n",
    "class MyLossNet(nn.Cell):\n",
    "    \"\"\"定义损失网络\"\"\"\n",
    "    def __init__(self, backbone, loss_fn): # backbone翻译为主干网络\n",
    "        super(MyLossNet, self).__init__(auto_prefix=False)\n",
    "        self.backbone = backbone\n",
    "        self.loss_fn = loss_fn\n",
    "    \n",
    "    def construct(self, data, label):\n",
    "        # 将神经网络的输出输入到损失函数，返回损失值\n",
    "        out = self.backbone(data)\n",
    "        return self.loss_fn(out, label)\n",
    "        \n",
    "    def backbone_net(self):\n",
    "        return self.backbone\n",
    "    \n",
    "    \n",
    "class MyTrainStep(nn.TrainOneStepCell):\n",
    "    \"\"\"定义模型训练过程\"\"\"\n",
    "    def __init__(self, network, optimizer):\n",
    "        # 初始化参数\n",
    "        super(MyTrainStep, self).__init__(network, optimizer)\n",
    "        self.grad = ops.GradOperation(get_by_list=True)\n",
    "        \n",
    "    def construct(self, data, label):\n",
    "        # 构建训练过程\n",
    "        weights = self.weights\n",
    "        loss = self.network(data, label)\n",
    "        grads = self.grad(self.network, weights)(data, label)\n",
    "        return loss, self.optimizer(grads)\n",
    "    \n",
    "# 执行训练\n",
    "loss_func = MyLossFunction()\n",
    "optim = MyOptimizer(0.01, net.trainable_params())\n",
    "loss_net = MyLossNet(net, loss_func)\n",
    "train_net = MyTrainStep(loss_net, optim)\n",
    "\n",
    "for data in train_dataset.create_dict_iterator():\n",
    "    train_net(data['data'], data['label'])\n",
    "    loss = loss_net(data['data'], data['label'])\n",
    "    print(f\"Loss: {loss}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43cc1e21",
   "metadata": {},
   "source": [
    "## 一、定义评价函数"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04fa2a63",
   "metadata": {},
   "source": [
    "mindspore.nn模块中提供了常见的评估函数，而当我们要自定义评估函数时，需要继承`nn.Metric`类，并重新实现其中的`clear`、`update`、`eval`方法。\n",
    "\n",
    "下面以平均绝对误差（MAE）来举例：\n",
    "\n",
    "$$ MAE=\\frac{1}{n}\\sum_{i=1}^n\\lvert y\\_pred_i - y_i \\rvert $$\n",
    "\n",
    "- `clear`：初始化相关的内部参数。\n",
    "- `update`：接收网络预测输出和标签，计算误差，并更新内部评估结果。一般在每个step后进行计算，并更新统计值。\n",
    "- `eval`：计算最终评估结果，一般在一个epoch结束后计算最终的评估结果。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e430bf96",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyMAE(nn.Metric):\n",
    "    \"\"\"定义评价指标\"\"\"\n",
    "    def __init__(self):\n",
    "        super(MyMAE, self).__init__()\n",
    "        self.clear()\n",
    "    \n",
    "    def clear(self):\n",
    "        # 初始化参数\n",
    "        self.abs_error_sum = 0\n",
    "        self.samples_num = 0\n",
    "    \n",
    "    def update(self, *inputs):\n",
    "        # 更新参数\n",
    "        y_predict = inputs[0].asnumpy()\n",
    "        y = inputs[1].asnumpy()\n",
    "        \n",
    "        # 计算绝对误差\n",
    "        error_abs = np.abs(y.reshape(y_predict.shape) - y_predict)\n",
    "        self.abs_error_sum += error_abs.sum()\n",
    "        self.samples_num += y.shape[0]\n",
    "        \n",
    "    def eval(self):\n",
    "        # 计算最终评估结果\n",
    "        return self.abs_error_sum / self.samples_num"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a392ddaa",
   "metadata": {},
   "source": [
    "## 二、定义评估网络\n",
    "同样的，评价网络需继承nn.Cell类。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "124eaedf",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyEvalNet(nn.Cell):\n",
    "    \"\"\"定义评估网络\"\"\"\n",
    "    def __init__(self, network):\n",
    "        super(MyEvalNet, self).__init__()\n",
    "        self.network = network\n",
    "        \n",
    "    def construct(self, data, label):\n",
    "        outputs = self.network(data)\n",
    "        return outputs, label"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1712ff3",
   "metadata": {},
   "source": [
    "## 三、执行推理并估计"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b04063ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WARNING] ME(27752:3544,MainProcess):2022-10-17-09:25:12.252.054 [mindspore\\dataset\\engine\\datasets_user_defined.py:656] Python multiprocessing is not supported on Windows platform.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE: 0.961028516292572\n"
     ]
    }
   ],
   "source": [
    "# 加载数据集\n",
    "eval_dataset = create_dataset(data_num=160, batch_size=16, repeat_num=1)\n",
    "\n",
    "# 初始化网络\n",
    "eval_net = MyEvalNet(net)\n",
    "eval_net.set_train(False)\n",
    "mae = MyMAE()\n",
    "\n",
    "# 执行推理\n",
    "for data in eval_dataset.create_dict_iterator():\n",
    "    output, y_eval = eval_net(data['data'], data['label'])\n",
    "    mae.update(output, y_eval)\n",
    "    \n",
    "acc = mae.eval()\n",
    "print(\"MAE:\", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c32e491",
   "metadata": {},
   "source": [
    "## 四、保存与导出\n",
    "这与我们初级课程的内容一致,我们查看一下训练后的w和b的值。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ced84a44",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "network.layer.weight : [[2.1115189]]\n",
      "network.layer.bias : [3.0326443]\n"
     ]
    }
   ],
   "source": [
    "ms.save_checkpoint(net, \"./linear.ckpt\")\n",
    "param_dict = ms.load_checkpoint(\"./linear.ckpt\")\n",
    "\n",
    "for i in param_dict:\n",
    "    print(i,\":\", param_dict[i].asnumpy())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mindspore",
   "language": "python",
   "name": "mindvision"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
